{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 导入所需模块\n",
    "import json\n",
    "import time\n",
    "from time import sleep\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "from bs4 import BeautifulSoup\n",
    "from requests_html import HTMLSession,UserAgent\n",
    "import random\n",
    "import os"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_header():\n",
    "    import fake_useragent\n",
    "    location = os.getcwd() + '/fake_useragent.json'\n",
    "    ua = fake_useragent.UserAgent(path=location)\n",
    "    return ua.random"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 专业数据_专科(高职)部分\n",
    "def get_size(page=1):\n",
    "    url=\"https://api.eol.cn/gkcx/api/?access_token=&keyword=&level1=2&level2=&page={0}&signsafe=&size=30&sort=view_total&uri=apidata/api/gkv3/special/lists\"\\\n",
    "        .format(page)\n",
    "    # 创建HTML会话对象\n",
    "    session = HTMLSession()\n",
    "    # user_agent = UserAgent().random  #创建随机请求头\n",
    "    user_agent = get_header()\n",
    "    header = {\"User-Agent\": user_agent}\n",
    "    res = session.post(url, headers=header)\n",
    "    data = json.loads(res.text)\n",
    "    size = 0\n",
    "    if data[\"message\"] == '成功':\n",
    "        size = data[\"data\"][\"numFound\"]\n",
    "    return size\n",
    "\n",
    "def get_university_info(size, page_size=30):\n",
    "    page_cnt = int(size/page_size) if size%page_size==0 else int(size/page_size)+1\n",
    "    print('一共{0}页数据，即将开始爬取...'.format(page_cnt))\n",
    "    session2 = HTMLSession()   #创建HTML会话对象\n",
    "    df_result = pd.DataFrame()\n",
    "    for index in range(1, page_cnt+1):\n",
    "        print('正在爬取第 {0}/{1} 页数据'.format(index, page_cnt))\n",
    "        url=\"https://api.eol.cn/gkcx/api/?access_token=&keyword=&level1=2&level2=&page={0}&signsafe=&size=30&sort=view_total&uri=apidata/api/gkv3/special/lists\"\\\n",
    "            .format(index)   \n",
    "        user_agent = UserAgent().random  #创建随机请求头\n",
    "        header = {\"User-Agent\": user_agent}\n",
    "        res = session2.post(url, headers=header)\n",
    "        data = json.loads(res.text)\n",
    "\n",
    "        if data[\"message\"] == '成功':\n",
    "            df_data = pd.DataFrame(data[\"data\"][\"item\"])\n",
    "            df_result = df_result.append(df_data)\n",
    "            time.sleep(random.randint(5, 7))\n",
    "\n",
    "    return df_result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "一共26页数据，即将开始爬取...\n",
      "正在爬取第 1/26 页数据\n",
      "正在爬取第 2/26 页数据\n",
      "正在爬取第 3/26 页数据\n",
      "正在爬取第 4/26 页数据\n",
      "正在爬取第 5/26 页数据\n",
      "正在爬取第 6/26 页数据\n",
      "正在爬取第 7/26 页数据\n",
      "正在爬取第 8/26 页数据\n",
      "正在爬取第 9/26 页数据\n",
      "正在爬取第 10/26 页数据\n",
      "正在爬取第 11/26 页数据\n",
      "正在爬取第 12/26 页数据\n",
      "正在爬取第 13/26 页数据\n",
      "正在爬取第 14/26 页数据\n",
      "正在爬取第 15/26 页数据\n",
      "正在爬取第 16/26 页数据\n",
      "正在爬取第 17/26 页数据\n",
      "正在爬取第 18/26 页数据\n",
      "正在爬取第 19/26 页数据\n",
      "正在爬取第 20/26 页数据\n",
      "正在爬取第 21/26 页数据\n",
      "正在爬取第 22/26 页数据\n",
      "正在爬取第 23/26 页数据\n",
      "正在爬取第 24/26 页数据\n",
      "正在爬取第 25/26 页数据\n",
      "正在爬取第 26/26 页数据\n"
     ]
    }
   ],
   "source": [
    "size = get_size()\n",
    "df_result = get_university_info(size)\n",
    "df_result.to_csv('专业数据_专科(高职).csv', encoding='gbk', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 专业数据_本科部分\n",
    "def get_size(page=1):\n",
    "    url=\"https://api.eol.cn/gkcx/api/?access_token=&keyword=&level1=1&level2=&page={0}&signsafe=&size=30&sort=view_total&uri=apidata/api/gkv3/special/lists\"\\\n",
    "        .format(page)\n",
    "    session = HTMLSession()   #创建HTML会话对象\n",
    "    user_agent = UserAgent().random  #创建随机请求头\n",
    "    header = {\"User-Agent\": user_agent}\n",
    "    res = session.post(url, headers=header)\n",
    "    data = json.loads(res.text)\n",
    "\n",
    "    size = 0\n",
    "    if data[\"message\"] == '成功':\n",
    "        size = data[\"data\"][\"numFound\"]\n",
    "    return size\n",
    "\n",
    "def get_university_info(size, page_size=30):\n",
    "    page_cnt = int(size/page_size) if size%page_size==0 else int(size/page_size)+1\n",
    "    print('一共{0}页数据，即将开始爬取...'.format(page_cnt))\n",
    "    session2 = HTMLSession()   #创建HTML会话对象\n",
    "    df_result = pd.DataFrame()\n",
    "    for index in range(1, page_cnt+1):\n",
    "        print('正在爬取第 {0}/{1} 页数据'.format(index, page_cnt))\n",
    "        url=\"https://api.eol.cn/gkcx/api/?access_token=&keyword=&level1=1&level2=&page={0}&signsafe=&size=30&sort=view_total&uri=apidata/api/gkv3/special/lists\"\\\n",
    "            .format(index)   \n",
    "        user_agent = UserAgent().random  #创建随机请求头\n",
    "        header = {\"User-Agent\": user_agent}\n",
    "        res = session2.post(url, headers=header)\n",
    "        data = json.loads(res.text)\n",
    "\n",
    "        if data[\"message\"] == '成功':\n",
    "            df_data = pd.DataFrame(data[\"data\"][\"item\"])\n",
    "            df_result = df_result.append(df_data)\n",
    "            time.sleep(random.randint(5, 7))\n",
    "\n",
    "    return df_result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "一共25页数据，即将开始爬取...\n",
      "正在爬取第 1/25 页数据\n",
      "正在爬取第 2/25 页数据\n",
      "正在爬取第 3/25 页数据\n",
      "正在爬取第 4/25 页数据\n",
      "正在爬取第 5/25 页数据\n",
      "正在爬取第 6/25 页数据\n",
      "正在爬取第 7/25 页数据\n",
      "正在爬取第 8/25 页数据\n",
      "正在爬取第 9/25 页数据\n",
      "正在爬取第 10/25 页数据\n",
      "正在爬取第 11/25 页数据\n",
      "正在爬取第 12/25 页数据\n",
      "正在爬取第 13/25 页数据\n",
      "正在爬取第 14/25 页数据\n",
      "正在爬取第 15/25 页数据\n",
      "正在爬取第 16/25 页数据\n",
      "正在爬取第 17/25 页数据\n",
      "正在爬取第 18/25 页数据\n",
      "正在爬取第 19/25 页数据\n",
      "正在爬取第 20/25 页数据\n",
      "正在爬取第 21/25 页数据\n",
      "正在爬取第 22/25 页数据\n",
      "正在爬取第 23/25 页数据\n",
      "正在爬取第 24/25 页数据\n",
      "正在爬取第 25/25 页数据\n"
     ]
    }
   ],
   "source": [
    "size = get_size()\n",
    "df_result = get_university_info(size)\n",
    "df_result.to_csv('专业数据_本科.csv', encoding='gbk', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 高校数据\n",
    "def get_size(page=1):\n",
    "    url = 'https://api.eol.cn/gkcx/api/?access_token=&admissions=&central=&department=&dual_class=&f211=&f985=&is_doublehigh=&is_dual_class=&keyword=&nature=&page={0}&province_id=&ranktype=&request_type=1&school_type=&signsafe=&size=20&sort=view_total&top_school_id=[2941]&type=&uri=apidata/api/gk/school/lists'\\\n",
    "        .format(page)\n",
    "    session = HTMLSession()   #创建HTML会话对象\n",
    "    user_agent = UserAgent().random  #创建随机请求头\n",
    "    header = {\"User-Agent\": user_agent}\n",
    "    res = session.post(url, headers=header)\n",
    "    data = json.loads(res.text)\n",
    "    size = 0\n",
    "    if data[\"message\"] == '成功':\n",
    "        size = data[\"data\"][\"numFound\"]\n",
    "    return size\n",
    "\n",
    "def get_university_info(size, page_size=20):\n",
    "    page_cnt = int(size/page_size) if size%page_size==0 else int(size/page_size)+1\n",
    "    print('一共{0}页数据，即将开始爬取...'.format(page_cnt))\n",
    "    session2 = HTMLSession()   #创建HTML会话对象\n",
    "    df_result = pd.DataFrame()\n",
    "    for index in range(1, page_cnt+1):\n",
    "        print('正在爬取第 {0}/{1} 页数据'.format(index, page_cnt))\n",
    "        url = 'https://api.eol.cn/gkcx/api/?access_token=&admissions=&central=&department=&dual_class=&f211=&f985=&is_doublehigh=&is_dual_class=&keyword=&nature=&page={0}&province_id=&ranktype=&request_type=1&school_type=&signsafe=&size=20&sort=view_total&top_school_id=[2941]&type=&uri=apidata/api/gk/school/lists' \\\n",
    "            .format(index)    \n",
    "        user_agent = UserAgent().random  #创建随机请求头\n",
    "        header = {\"User-Agent\": user_agent}\n",
    "        res = session2.post(url, headers=header)\n",
    "        data = json.loads(res.text)\n",
    "\n",
    "        if data[\"message\"] == '成功':\n",
    "            df_data = pd.DataFrame(data[\"data\"][\"item\"])\n",
    "            df_result = df_result.append(df_data)\n",
    "            time.sleep(random.randint(5, 7))\n",
    "\n",
    "    return df_result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "一共143页数据，即将开始爬取...\n",
      "正在爬取第 1/143 页数据\n",
      "正在爬取第 2/143 页数据\n",
      "正在爬取第 3/143 页数据\n",
      "正在爬取第 4/143 页数据\n",
      "正在爬取第 5/143 页数据\n",
      "正在爬取第 6/143 页数据\n",
      "正在爬取第 7/143 页数据\n",
      "正在爬取第 8/143 页数据\n",
      "正在爬取第 9/143 页数据\n",
      "正在爬取第 10/143 页数据\n",
      "正在爬取第 11/143 页数据\n",
      "正在爬取第 12/143 页数据\n",
      "正在爬取第 13/143 页数据\n",
      "正在爬取第 14/143 页数据\n",
      "正在爬取第 15/143 页数据\n",
      "正在爬取第 16/143 页数据\n",
      "正在爬取第 17/143 页数据\n",
      "正在爬取第 18/143 页数据\n",
      "正在爬取第 19/143 页数据\n",
      "正在爬取第 20/143 页数据\n",
      "正在爬取第 21/143 页数据\n",
      "正在爬取第 22/143 页数据\n",
      "正在爬取第 23/143 页数据\n",
      "正在爬取第 24/143 页数据\n",
      "正在爬取第 25/143 页数据\n",
      "正在爬取第 26/143 页数据\n",
      "正在爬取第 27/143 页数据\n",
      "正在爬取第 28/143 页数据\n",
      "正在爬取第 29/143 页数据\n",
      "正在爬取第 30/143 页数据\n",
      "正在爬取第 31/143 页数据\n",
      "正在爬取第 32/143 页数据\n",
      "正在爬取第 33/143 页数据\n",
      "正在爬取第 34/143 页数据\n",
      "正在爬取第 35/143 页数据\n",
      "正在爬取第 36/143 页数据\n",
      "正在爬取第 37/143 页数据\n",
      "正在爬取第 38/143 页数据\n",
      "正在爬取第 39/143 页数据\n",
      "正在爬取第 40/143 页数据\n",
      "正在爬取第 41/143 页数据\n",
      "正在爬取第 42/143 页数据\n",
      "正在爬取第 43/143 页数据\n",
      "正在爬取第 44/143 页数据\n",
      "正在爬取第 45/143 页数据\n",
      "正在爬取第 46/143 页数据\n",
      "正在爬取第 47/143 页数据\n",
      "正在爬取第 48/143 页数据\n",
      "正在爬取第 49/143 页数据\n",
      "正在爬取第 50/143 页数据\n",
      "正在爬取第 51/143 页数据\n",
      "正在爬取第 52/143 页数据\n",
      "正在爬取第 53/143 页数据\n",
      "正在爬取第 54/143 页数据\n",
      "正在爬取第 55/143 页数据\n",
      "正在爬取第 56/143 页数据\n",
      "正在爬取第 57/143 页数据\n",
      "正在爬取第 58/143 页数据\n",
      "正在爬取第 59/143 页数据\n",
      "正在爬取第 60/143 页数据\n",
      "正在爬取第 61/143 页数据\n",
      "正在爬取第 62/143 页数据\n",
      "正在爬取第 63/143 页数据\n",
      "正在爬取第 64/143 页数据\n",
      "正在爬取第 65/143 页数据\n",
      "正在爬取第 66/143 页数据\n",
      "正在爬取第 67/143 页数据\n",
      "正在爬取第 68/143 页数据\n",
      "正在爬取第 69/143 页数据\n",
      "正在爬取第 70/143 页数据\n",
      "正在爬取第 71/143 页数据\n",
      "正在爬取第 72/143 页数据\n",
      "正在爬取第 73/143 页数据\n",
      "正在爬取第 74/143 页数据\n",
      "正在爬取第 75/143 页数据\n",
      "正在爬取第 76/143 页数据\n",
      "正在爬取第 77/143 页数据\n",
      "正在爬取第 78/143 页数据\n",
      "正在爬取第 79/143 页数据\n",
      "正在爬取第 80/143 页数据\n",
      "正在爬取第 81/143 页数据\n",
      "正在爬取第 82/143 页数据\n",
      "正在爬取第 83/143 页数据\n",
      "正在爬取第 84/143 页数据\n",
      "正在爬取第 85/143 页数据\n",
      "正在爬取第 86/143 页数据\n",
      "正在爬取第 87/143 页数据\n",
      "正在爬取第 88/143 页数据\n",
      "正在爬取第 89/143 页数据\n",
      "正在爬取第 90/143 页数据\n",
      "正在爬取第 91/143 页数据\n",
      "正在爬取第 92/143 页数据\n",
      "正在爬取第 93/143 页数据\n",
      "正在爬取第 94/143 页数据\n",
      "正在爬取第 95/143 页数据\n",
      "正在爬取第 96/143 页数据\n",
      "正在爬取第 97/143 页数据\n",
      "正在爬取第 98/143 页数据\n",
      "正在爬取第 99/143 页数据\n",
      "正在爬取第 100/143 页数据\n",
      "正在爬取第 101/143 页数据\n",
      "正在爬取第 102/143 页数据\n",
      "正在爬取第 103/143 页数据\n",
      "正在爬取第 104/143 页数据\n",
      "正在爬取第 105/143 页数据\n",
      "正在爬取第 106/143 页数据\n",
      "正在爬取第 107/143 页数据\n",
      "正在爬取第 108/143 页数据\n",
      "正在爬取第 109/143 页数据\n",
      "正在爬取第 110/143 页数据\n",
      "正在爬取第 111/143 页数据\n",
      "正在爬取第 112/143 页数据\n",
      "正在爬取第 113/143 页数据\n",
      "正在爬取第 114/143 页数据\n",
      "正在爬取第 115/143 页数据\n",
      "正在爬取第 116/143 页数据\n",
      "正在爬取第 117/143 页数据\n",
      "正在爬取第 118/143 页数据\n",
      "正在爬取第 119/143 页数据\n",
      "正在爬取第 120/143 页数据\n",
      "正在爬取第 121/143 页数据\n",
      "正在爬取第 122/143 页数据\n",
      "正在爬取第 123/143 页数据\n",
      "正在爬取第 124/143 页数据\n",
      "正在爬取第 125/143 页数据\n",
      "正在爬取第 126/143 页数据\n",
      "正在爬取第 127/143 页数据\n",
      "正在爬取第 128/143 页数据\n",
      "正在爬取第 129/143 页数据\n",
      "正在爬取第 130/143 页数据\n",
      "正在爬取第 131/143 页数据\n",
      "正在爬取第 132/143 页数据\n",
      "正在爬取第 133/143 页数据\n",
      "正在爬取第 134/143 页数据\n",
      "正在爬取第 135/143 页数据\n",
      "正在爬取第 136/143 页数据\n",
      "正在爬取第 137/143 页数据\n",
      "正在爬取第 138/143 页数据\n",
      "正在爬取第 139/143 页数据\n",
      "正在爬取第 140/143 页数据\n",
      "正在爬取第 141/143 页数据\n",
      "正在爬取第 142/143 页数据\n",
      "正在爬取第 143/143 页数据\n"
     ]
    }
   ],
   "source": [
    "size = get_size()\n",
    "df_result = get_university_info(size)\n",
    "df_result.to_csv('高校数据.csv', encoding='gbk', index=False)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.6rc1"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
